import re
from copy import deepcopy

import scrapy


class MtySpider(scrapy.Spider):
    name = 'mty'
    allowed_domains = ['tripadvisor.cn']
    start_urls = ['https://www.tripadvisor.cn']

    def start_requests(self):

        cookie_str = """
        """
        # cookies_dict = {i.split("=")[0]: i.split("=")[1] for i in cookie_str.split("; ")}

        # 国家地区url
        url = "https://www.tripadvisor.cn/Lvyou"
        yield scrapy.Request(
            url,
            callback=self.parse,
            # cookies=cookies_dict,
            # headers={}
        )

    def parse(self, response):
        li_list = response.xpath("//div[@id='tab-body-wrapper']/div[@class='tab-body']/ul/li")
        for li in li_list:
            hotel = {}
            hotel["location"] = {}
            hotel["location"]["country"] = li.xpath("./a/text()").extract_first()
            hotel_country_url = self.start_urls[0] + li.xpath("./a/@href").extract_first()

            # hotel["location"]["country_url"] = hotel_country_url
            yield scrapy.Request(
                hotel_country_url,
                callback=self.parse_location,
                meta={"item": deepcopy(hotel)},
                headers={
                    "Referer": response.url
                }
            )

    def parse_location(self, response):
        """酒店链接入口"""
        hotel = response.meta["item"]
        country_hotel_url = response.xpath("//div[@id='component_6']//div/span[1]/div/a/@href").extract_first()

        if country_hotel_url:
            country_hotel_full_url = self.start_urls[0] + country_hotel_url
            yield scrapy.Request(
                country_hotel_full_url,
                callback=self.parse_hotel_list_part1,
                meta={"item": deepcopy(hotel)},
                headers={
                    "Referer": response.url
                }
            )

    def parse_hotel_list_part1(self, response):
        """hotel城市第一页"""
        hotel = response.meta["item"]
        div_list = response.xpath("//div[@id='taplc_hotels_leaf_geo_list_dusty_hotels_resp_0']//div[@class='geo_wrap']")
        for div in div_list:
            city_hotel_url = self.start_urls[0] + div.xpath("./a/@href").extract_first()
            # hotel["location"]["city_hotel_url"] = city_hotel_url
            yield scrapy.Request(
                city_hotel_url,
                callback=self.parse_hotel_detail_list,
                meta={"item": deepcopy(hotel)},
                headers={
                    "Referer": response.url
                }
            )
        # 第二页开始url变化，数据提取变化
        # next_url = "https://www.tripadvisor.cn/Hotels-g294211-oa20-China-Hotels.html"
        next_url = response.xpath('//a[text()="下一页"]/@href').extract_first()
        if next_url is not None:
            next_full_url = self.start_urls[0] + next_url
            yield scrapy.Request(
                next_full_url,
                callback=self.parse_hotel_city_list_next_part,
                meta={"item": deepcopy(hotel)},
                headers={
                    "Referer": response.url
                }
            )

    def parse_hotel_city_list_next_part(self, response):
        """hotel城市第二页"""
        hotel = response.meta["item"]
        # total_page = response.xpath('//div[@id="taplc_broad_geo_tiles_dusty_hotels_resp_0"]//div[@class="pageNumbers"]/a[6]/text()').extract_first()
        li_list = response.xpath("//div[@id='taplc_broad_geo_tiles_dusty_hotels_resp_0']/ul/li")
        for li in li_list:
            hotel_city_url = self.start_urls[0] + li.xpath("./a/@href").extract_first()
            yield scrapy.Request(
                hotel_city_url,
                callback=self.parse_hotel_detail_list,
                meta={"item": deepcopy(hotel)},
                headers={
                    "Referer": response.url
                }
            )

        # hotel_url第二页以后翻页
        # next_url = response.xpath('//div[@id="taplc_broad_geo_tiles_dusty_hotels_resp_0"]//div[@class="pagination_wrapper"]//a[text()="下一页"]/@href').extract_first()
        next_url = response.xpath('//a[text()="下一页"]/@href').extract_first()
        if next_url is not None:
            next_full_url = self.start_urls[0] + next_url
            # yield response.follow(
            yield scrapy.Request(
                next_full_url,
                callback=self.parse_hotel_city_list_next_part,
                meta={"item": deepcopy(hotel)},
                headers={
                    "Referer": response.url
                }
            )

    def parse_hotel_detail_list(self, response):
        """该城市酒店列表"""
        hotel = response.meta["item"]
        # print(">>>>>>>>>>>",response.url)

        li_list = response.xpath("//div[@id='taplc_trip_planner_breadcrumbs_0']/ul")
        for li in li_list:
            hotel["location"]["continent"] = li.xpath("./li[1]/a/span/text()").extract_first()
            # hotel["continent_url"] = self.start_urls[0] + li.xpath("./li[1]/a/@href").extract_first()
            hotel["location"]["province"] = li.xpath("./li[3]/a/span/text()").extract_first()
            # hotel["province_url"] = self.start_urls[0] + li.xpath("./li[3]/a/@href").extract_first()
            hotel["location"]["city"] = li.xpath("./li[4]/a/span/text()").extract_first()
            # hotel["city_url"] = self.start_urls[0] + li.xpath("./li[4]/a/@href").extract_first()
            # print(hotel)


        div_hotel_list = response.xpath("//div[@id='taplc_hsx_hotel_list_lite_dusty_hotels_combined_sponsored_0']/div")  # 此处div[position()>1]为什么不能进入下一步
        for div in div_hotel_list:
            hotel["hotel_info"] = {}
            hotel["hotel_info"]["name"] = div.xpath(".//div[@class='listing_title']/a/text()").extract_first()
            # hotel["hotel_rank"] = div.xpath(".//div[@class='popindex']/text()").extract_first()
            hotel_url = div.xpath(".//div[@class='listing_title']/a/@href").extract_first()
            if hotel_url is not None:
                hotel_detail_url = self.start_urls[0] + hotel_url
                hotel["hotel_info"]["hotel_detail_url"] = hotel_detail_url
                yield scrapy.Request(
                    # yield response.follow(
                    hotel_detail_url,
                    callback=self.parse_hotel_detail_info,
                    meta={"item": deepcopy(hotel)},
                    headers={
                        "Referer": response.url
                    }
                )

    def parse_hotel_detail_info(self, response):
        hotel = response.meta["item"]
        """
        响应式小页面可点击提取
        """

    #     hotel["hotel_info"]["hotel_english_name"] = response.xpath('//h1[@id="HEADING"]/div/text()').extract_first()
    #     hotel["hotel_info"]["hotel_address"] = response.xpath(
    #         '//div[@id="component_4"]//span[@class="public-business-listing-ContactInfo__ui_link--1_7Zp public-business-listing-ContactInfo__level_4--3JgmI"]/text()').extract_first()
    #     hotel["hotel_info"]["hotel_phone"] = response.xpath(
    #         '//div[@id="component_4"]/div/div/div[2]/div/div[2]/div/div[2]/div/a/span[2]/text()').extract_first()
    #     hotel["hotel_info"]["hotel_rank"] = response.xpath(
    #         '//*[@id="ABOUT_TAB"]/div/div[1]/span/text()').extract_first()

        # 页面不同，提取方式不同
        # print(">>>>>>>>>>>>",response.url)

        div_origin_list = response.xpath('//*[@id="component_5"]/div/div/div')
        origin_list = []

        if not div_origin_list.xpath("./a"):
            op_list = []
            for div in div_origin_list:
                origin = div.xpath('.//div/img/@alt').extract_first()
                price = div.xpath('./div[1]/div[2]//div/text()').extract_first()
                if origin and price:
                    op_list.append({origin:price})
            origin_list += op_list

        if div_origin_list.xpath("./a"):
            origin_a = div_origin_list.xpath('./a/div/div[1]/img/@alt').extract_first()
            price_a = div_origin_list.xpath('./a/div/div[2]/div/text()').extract_first()
            if origin_a and price_a:
                origin_list.append({origin_a: price_a})

        # 每种页面都有或者没有此种情况
        sub_div_origin_list = response.xpath('//*[@id="component_5"]//div[@class="ui_columns is-multiline is-gapless-vertical"]/div')
        if sub_div_origin_list:
            sop_list = []
            for sub in sub_div_origin_list:
                sub_origin = sub.xpath('./div/span/span/text()').extract_first()
                sub_price = sub.xpath('./div/span/text()').extract_first()
                if sub_origin and sub_price:
                    sop_list.append({sub_origin: sub_price})
            origin_list += sop_list


        # print(origin_list)
        # hotel["hotel_info"]["origin"] = origin_list

        # origin = response.xpath('//div[@id="component_4"]/div/di').extract_first()
        # origin = response.xpath('//div[@id="component_4"]/div/di').extract_first()
        # origin = response.xpath('//div[@id="component_4"]/div/di').extract_first()

        # print(hotel)
